{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Age, Gender and Emotion Detection\n",
    "\n",
    "### Let's load our classfiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import cv2\n",
    "import dlib\n",
    "import sys\n",
    "import numpy as np\n",
    "import argparse\n",
    "from contextlib import contextmanager\n",
    "from wide_resnet import WideResNet\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.models import load_model\n",
    "from keras.preprocessing.image import img_to_array\n",
    "\n",
    "classifier = load_model('/home/deeplearningcv/DeepLearningCV/Trained Models/emotion_little_vgg_2.h5')\n",
    "face_classifier = cv2.CascadeClassifier('./Haarcascades/haarcascade_frontalface_default.xml')\n",
    "pretrained_model = \"https://github.com/yu4u/age-gender-estimation/releases/download/v0.5/weights.28-3.73.hdf5\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing our Emotion, Age and Gender Detector - Using Webcam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "modhash = 'fbe63257a054c1c5466cfd7bf14646d6'\n",
    "emotion_classes = {0: 'Angry', 1: 'Fear', 2: 'Happy', 3: 'Neutral', 4: 'Sad', 5: 'Surprise'}\n",
    "\n",
    "def face_detector(img):\n",
    "    # Convert image to grayscale for faster detection\n",
    "    gray = cv2.cvtColor(img.copy(),cv2.COLOR_BGR2GRAY)\n",
    "    faces = face_classifier.detectMultiScale(gray, 1.3, 5)\n",
    "    if faces is ():\n",
    "        return False ,(0,0,0,0), np.zeros((1,48,48,3), np.uint8), img\n",
    "    \n",
    "    allfaces = []   \n",
    "    rects = []\n",
    "    for (x,y,w,h) in faces:\n",
    "        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)\n",
    "        roi = img[y:y+h, x:x+w]\n",
    "        allfaces.append(roi)\n",
    "        rects.append((x,w,y,h))\n",
    "    return True, rects, allfaces, img\n",
    "\n",
    "# Define our model parameters\n",
    "depth = 16\n",
    "k = 8\n",
    "weight_file = None\n",
    "margin = 0.4\n",
    "image_dir = None\n",
    "\n",
    "# Get our weight file \n",
    "if not weight_file:\n",
    "    weight_file = get_file(\"weights.28-3.73.hdf5\", pretrained_model, cache_subdir=\"pretrained_models\",\n",
    "                           file_hash=modhash, cache_dir=Path(sys.argv[0]).resolve().parent)\n",
    "\n",
    "# load model and weights\n",
    "img_size = 64\n",
    "model = WideResNet(img_size, depth=depth, k=k)()\n",
    "model.load_weights(weight_file)\n",
    "\n",
    "# Initialize Webcam\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "while True:\n",
    "    ret, frame = cap.read()\n",
    "    ret, rects, faces, image = face_detector(frame)\n",
    "    preprocessed_faces_ag = []\n",
    "    preprocessed_faces_emo = []\n",
    "    \n",
    "    if ret:\n",
    "        for (i,face) in enumerate(faces):\n",
    "            face_ag = cv2.resize(face, (64, 64), interpolation = cv2.INTER_AREA)\n",
    "            preprocessed_faces_ag.append(face_ag)\n",
    "\n",
    "            face_gray_emo = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)\n",
    "            face_gray_emo = cv2.resize(face_gray_emo, (48, 48), interpolation = cv2.INTER_AREA)\n",
    "            face_gray_emo = face_gray_emo.astype(\"float\") / 255.0\n",
    "            face_gray_emo = img_to_array(face_gray_emo)\n",
    "            face_gray_emo = np.expand_dims(face_gray_emo, axis=0)\n",
    "            preprocessed_faces_emo.append(face_gray_emo)\n",
    "            \n",
    "        # make a prediction for Age and Gender\n",
    "        results = model.predict(np.array(preprocessed_faces_ag))\n",
    "        predicted_genders = results[0]\n",
    "        ages = np.arange(0, 101).reshape(101, 1)\n",
    "        predicted_ages = results[1].dot(ages).flatten()\n",
    "\n",
    "        # make a prediction for Emotion \n",
    "        emo_labels = []\n",
    "        for (i, face) in enumerate(faces):\n",
    "            preds = classifier.predict(preprocessed_faces_emo[i])[0]\n",
    "            emo_labels.append(emotion_classes[preds.argmax()])\n",
    "        \n",
    "        # draw results, for Age and Gender\n",
    "        for (i, face) in enumerate(faces):\n",
    "            label = \"{}, {}, {}\".format(int(predicted_ages[i]),\n",
    "                                        \"F\" if predicted_genders[i][0] > 0.6 else \"M\",\n",
    "                                        emo_labels[i])\n",
    "            \n",
    "        #Overlay our detected emotion on our pic\n",
    "        for (i, face) in enumerate(faces):\n",
    "            label_position = (rects[i][0] + int((rects[i][1]/2)), abs(rects[i][2] - 10))\n",
    "            cv2.putText(image, label, label_position , cv2.FONT_HERSHEY_PLAIN,1, (0,255,0), 2)\n",
    "\n",
    "    cv2.imshow(\"Emotion Detector\", image)\n",
    "    if cv2.waitKey(1) == 13: #13 is the Enter Key\n",
    "        break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap.release()\n",
    "cv2.destroyAllWindows()      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing our Emotion, Age and Gender Detector - On Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "# Define Image Path Here\n",
    "image_path = \"./images/\"\n",
    "\n",
    "modhash = 'fbe63257a054c1c5466cfd7bf14646d6'\n",
    "emotion_classes = {0: 'Angry', 1: 'Fear', 2: 'Happy', 3: 'Neutral', 4: 'Sad', 5: 'Surprise'}\n",
    "\n",
    "def face_detector(img):\n",
    "    # Convert image to grayscale for faster detection\n",
    "    gray = cv2.cvtColor(img.copy(),cv2.COLOR_BGR2GRAY)\n",
    "    faces = face_classifier.detectMultiScale(gray, 1.3, 5)\n",
    "    if faces is ():\n",
    "        return False ,(0,0,0,0), np.zeros((1,48,48,3), np.uint8), img\n",
    "    \n",
    "    allfaces = []   \n",
    "    rects = []\n",
    "    for (x,y,w,h) in faces:\n",
    "        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)\n",
    "        roi = img[y:y+h, x:x+w]\n",
    "        allfaces.append(roi)\n",
    "        rects.append((x,w,y,h))\n",
    "    return True, rects, allfaces, img\n",
    "\n",
    "# Define our model parameters\n",
    "depth = 16\n",
    "k = 8\n",
    "weight_file = None\n",
    "margin = 0.4\n",
    "image_dir = None\n",
    "\n",
    "# Get our weight file \n",
    "if not weight_file:\n",
    "    weight_file = get_file(\"weights.28-3.73.hdf5\", pretrained_model, cache_subdir=\"pretrained_models\",\n",
    "                           file_hash=modhash, cache_dir=Path(sys.argv[0]).resolve().parent)\n",
    "# load model and weights\n",
    "img_size = 64\n",
    "model = WideResNet(img_size, depth=depth, k=k)()\n",
    "model.load_weights(weight_file)\n",
    "\n",
    "image_names = [f for f in listdir(image_path) if isfile(join(image_path, f))]\n",
    "\n",
    "for image_name in image_names:\n",
    "    frame = cv2.imread(\"./images/\" + image_name)\n",
    "    ret, rects, faces, image = face_detector(frame)\n",
    "    preprocessed_faces_ag = []\n",
    "    preprocessed_faces_emo = []\n",
    "    \n",
    "    if ret:\n",
    "        for (i,face) in enumerate(faces):\n",
    "            face_ag = cv2.resize(face, (64, 64), interpolation = cv2.INTER_AREA)\n",
    "            preprocessed_faces_ag.append(face_ag)\n",
    "\n",
    "            face_gray_emo = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)\n",
    "            face_gray_emo = cv2.resize(face_gray_emo, (48, 48), interpolation = cv2.INTER_AREA)\n",
    "            face_gray_emo = face_gray_emo.astype(\"float\") / 255.0\n",
    "            face_gray_emo = img_to_array(face_gray_emo)\n",
    "            face_gray_emo = np.expand_dims(face_gray_emo, axis=0)\n",
    "            preprocessed_faces_emo.append(face_gray_emo)\n",
    "            \n",
    "        # make a prediction for Age and Gender\n",
    "        results = model.predict(np.array(preprocessed_faces_ag))\n",
    "        predicted_genders = results[0]\n",
    "        ages = np.arange(0, 101).reshape(101, 1)\n",
    "        predicted_ages = results[1].dot(ages).flatten()\n",
    "\n",
    "        # make a prediction for Emotion \n",
    "        emo_labels = []\n",
    "        for (i, face) in enumerate(faces):\n",
    "            preds = classifier.predict(preprocessed_faces_emo[i])[0]\n",
    "            emo_labels.append(emotion_classes[preds.argmax()])\n",
    "        \n",
    "        # draw results, for Age and Gender\n",
    "        for (i, face) in enumerate(faces):\n",
    "            label = \"{}, {}, {}\".format(int(predicted_ages[i]),\n",
    "                                        \"F\" if predicted_genders[i][0] > 0.4 else \"M\",\n",
    "                                        emo_labels[i])\n",
    "            \n",
    "        #Overlay our detected emotion on our pic\n",
    "        for (i, face) in enumerate(faces):\n",
    "            label_position = (rects[i][0] + int((rects[i][1]/2)), abs(rects[i][2] - 10))\n",
    "            cv2.putText(image, label, label_position , cv2.FONT_HERSHEY_PLAIN,1, (0,255,0), 2)\n",
    "\n",
    "    cv2.imshow(\"Emotion Detector\", image)\n",
    "    cv2.waitKey(0)\n",
    "\n",
    "cv2.destroyAllWindows()      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Dlib's Face Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "# Define Image Path Here\n",
    "image_path = \"./images/\"\n",
    "\n",
    "modhash = 'fbe63257a054c1c5466cfd7bf14646d6'\n",
    "emotion_classes = {0: 'Angry', 1: 'Fear', 2: 'Happy', 3: 'Neutral', 4: 'Sad', 5: 'Surprise'}\n",
    "\n",
    "def draw_label(image, point, label, font=cv2.FONT_HERSHEY_SIMPLEX,\n",
    "               font_scale=0.8, thickness=1):\n",
    "    size = cv2.getTextSize(label, font, font_scale, thickness)[0]\n",
    "    x, y = point\n",
    "    cv2.rectangle(image, (x, y - size[1]), (x + size[0], y), (255, 0, 0), cv2.FILLED)\n",
    "    cv2.putText(image, label, point, font, font_scale, (255, 255, 255), thickness, lineType=cv2.LINE_AA)\n",
    "    \n",
    "\n",
    "# Define our model parameters\n",
    "depth = 16\n",
    "k = 8\n",
    "weight_file = None\n",
    "margin = 0.4\n",
    "image_dir = None\n",
    "\n",
    "# Get our weight file \n",
    "if not weight_file:\n",
    "    weight_file = get_file(\"weights.28-3.73.hdf5\", pretrained_model, cache_subdir=\"pretrained_models\",\n",
    "                           file_hash=modhash, cache_dir=Path(sys.argv[0]).resolve().parent)\n",
    "# load model and weights\n",
    "img_size = 64\n",
    "model = WideResNet(img_size, depth=depth, k=k)()\n",
    "model.load_weights(weight_file)\n",
    "\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "\n",
    "image_names = [f for f in listdir(image_path) if isfile(join(image_path, f))]\n",
    "\n",
    "for image_name in image_names:\n",
    "    frame = cv2.imread(\"./images/\" + image_name)\n",
    "    preprocessed_faces_emo = []           \n",
    " \n",
    "    input_img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "    img_h, img_w, _ = np.shape(input_img)\n",
    "    detected = detector(frame, 1)\n",
    "    faces = np.empty((len(detected), img_size, img_size, 3))\n",
    "    \n",
    "    preprocessed_faces_emo = []\n",
    "    if len(detected) > 0:\n",
    "        for i, d in enumerate(detected):\n",
    "            x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, d.bottom() + 1, d.width(), d.height()\n",
    "            xw1 = max(int(x1 - margin * w), 0)\n",
    "            yw1 = max(int(y1 - margin * h), 0)\n",
    "            xw2 = min(int(x2 + margin * w), img_w - 1)\n",
    "            yw2 = min(int(y2 + margin * h), img_h - 1)\n",
    "            cv2.rectangle(frame, (x1, y1), (x2, y2), (255, 0, 0), 2)\n",
    "            # cv2.rectangle(img, (xw1, yw1), (xw2, yw2), (255, 0, 0), 2)\n",
    "            faces[i, :, :, :] = cv2.resize(frame[yw1:yw2 + 1, xw1:xw2 + 1, :], (img_size, img_size))\n",
    "            face =  frame[yw1:yw2 + 1, xw1:xw2 + 1, :]\n",
    "            face_gray_emo = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)\n",
    "            face_gray_emo = cv2.resize(face_gray_emo, (48, 48), interpolation = cv2.INTER_AREA)\n",
    "            face_gray_emo = face_gray_emo.astype(\"float\") / 255.0\n",
    "            face_gray_emo = img_to_array(face_gray_emo)\n",
    "            face_gray_emo = np.expand_dims(face_gray_emo, axis=0)\n",
    "            preprocessed_faces_emo.append(face_gray_emo)\n",
    "\n",
    "        # make a prediction for Age and Gender\n",
    "        results = model.predict(np.array(faces))\n",
    "        predicted_genders = results[0]\n",
    "        ages = np.arange(0, 101).reshape(101, 1)\n",
    "        predicted_ages = results[1].dot(ages).flatten()\n",
    "\n",
    "        # make a prediction for Emotion \n",
    "        emo_labels = []\n",
    "        for i, d in enumerate(detected):\n",
    "            preds = classifier.predict(preprocessed_faces_emo[i])[0]\n",
    "            emo_labels.append(emotion_classes[preds.argmax()])\n",
    "        \n",
    "        # draw results\n",
    "        for i, d in enumerate(detected):\n",
    "            label = \"{}, {}, {}\".format(int(predicted_ages[i]),\n",
    "                                        \"F\" if predicted_genders[i][0] > 0.4 else \"M\", emo_labels[i])\n",
    "            draw_label(frame, (d.left(), d.top()), label)\n",
    "\n",
    "    cv2.imshow(\"Emotion Detector\", frame)\n",
    "    cv2.waitKey(0)\n",
    "\n",
    "cv2.destroyAllWindows()      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### And now using dlib's detector with our webcam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "# Define Image Path Here\n",
    "image_path = \"./images/\"\n",
    "\n",
    "modhash = 'fbe63257a054c1c5466cfd7bf14646d6'\n",
    "emotion_classes = {0: 'Angry', 1: 'Fear', 2: 'Happy', 3: 'Neutral', 4: 'Sad', 5: 'Surprise'}\n",
    "\n",
    "def draw_label(image, point, label, font=cv2.FONT_HERSHEY_SIMPLEX,\n",
    "               font_scale=0.8, thickness=1):\n",
    "    size = cv2.getTextSize(label, font, font_scale, thickness)[0]\n",
    "    x, y = point\n",
    "    cv2.rectangle(image, (x, y - size[1]), (x + size[0], y), (255, 0, 0), cv2.FILLED)\n",
    "    cv2.putText(image, label, point, font, font_scale, (255, 255, 255), thickness, lineType=cv2.LINE_AA)\n",
    "    \n",
    "\n",
    "# Define our model parameters\n",
    "depth = 16\n",
    "k = 8\n",
    "weight_file = None\n",
    "margin = 0.4\n",
    "image_dir = None\n",
    "\n",
    "# Get our weight file \n",
    "if not weight_file:\n",
    "    weight_file = get_file(\"weights.28-3.73.hdf5\", pretrained_model, cache_subdir=\"pretrained_models\",\n",
    "                           file_hash=modhash, cache_dir=Path(sys.argv[0]).resolve().parent)\n",
    "# load model and weights\n",
    "img_size = 64\n",
    "model = WideResNet(img_size, depth=depth, k=k)()\n",
    "model.load_weights(weight_file)\n",
    "\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "\n",
    "# Initialize Webcam\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "while True:\n",
    "    ret, frame = cap.read()\n",
    "    preprocessed_faces_emo = []           \n",
    " \n",
    "    input_img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "    img_h, img_w, _ = np.shape(input_img)\n",
    "    detected = detector(frame, 1)\n",
    "    faces = np.empty((len(detected), img_size, img_size, 3))\n",
    "    \n",
    "    preprocessed_faces_emo = []\n",
    "    if len(detected) > 0:\n",
    "        for i, d in enumerate(detected):\n",
    "            x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, d.bottom() + 1, d.width(), d.height()\n",
    "            xw1 = max(int(x1 - margin * w), 0)\n",
    "            yw1 = max(int(y1 - margin * h), 0)\n",
    "            xw2 = min(int(x2 + margin * w), img_w - 1)\n",
    "            yw2 = min(int(y2 + margin * h), img_h - 1)\n",
    "            cv2.rectangle(frame, (x1, y1), (x2, y2), (255, 0, 0), 2)\n",
    "            # cv2.rectangle(img, (xw1, yw1), (xw2, yw2), (255, 0, 0), 2)\n",
    "            faces[i, :, :, :] = cv2.resize(frame[yw1:yw2 + 1, xw1:xw2 + 1, :], (img_size, img_size))\n",
    "            face =  frame[yw1:yw2 + 1, xw1:xw2 + 1, :]\n",
    "            face_gray_emo = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)\n",
    "            face_gray_emo = cv2.resize(face_gray_emo, (48, 48), interpolation = cv2.INTER_AREA)\n",
    "            face_gray_emo = face_gray_emo.astype(\"float\") / 255.0\n",
    "            face_gray_emo = img_to_array(face_gray_emo)\n",
    "            face_gray_emo = np.expand_dims(face_gray_emo, axis=0)\n",
    "            preprocessed_faces_emo.append(face_gray_emo)\n",
    "\n",
    "        # make a prediction for Age and Gender\n",
    "        results = model.predict(np.array(faces))\n",
    "        predicted_genders = results[0]\n",
    "        ages = np.arange(0, 101).reshape(101, 1)\n",
    "        predicted_ages = results[1].dot(ages).flatten()\n",
    "\n",
    "        # make a prediction for Emotion \n",
    "        emo_labels = []\n",
    "        for i, d in enumerate(detected):\n",
    "            preds = classifier.predict(preprocessed_faces_emo[i])[0]\n",
    "            emo_labels.append(emotion_classes[preds.argmax()])\n",
    "        \n",
    "        # draw results\n",
    "        for i, d in enumerate(detected):\n",
    "            label = \"{}, {}, {}\".format(int(predicted_ages[i]),\n",
    "                                        \"F\" if predicted_genders[i][0] > 0.4 else \"M\", emo_labels[i])\n",
    "            draw_label(frame, (d.left(), d.top()), label)\n",
    "\n",
    "    cv2.imshow(\"Emotion Detector\", frame)\n",
    "    if cv2.waitKey(1) == 13: #13 is the Enter Key\n",
    "        break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
